/**
 * 文件名:AuthorCrawler.java
 * 包名:com.wx.demo.wcd.crawler
 * 创建时间:2017年2月22日下午3:56:20
 * <p>Copyright ®医界互联 TUS 平台版权所有。</p>
 *
 */

package com.wx.demo.wcd.crawler;

import java.util.ArrayList;
import java.util.Date;

import org.apache.log4j.Logger;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.context.support.AbstractApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;

import com.wx.demo.wcd.domain.po.AuthorPO;
import com.wx.demo.wcd.pub.UUIDGenerator;
import com.wx.demo.wcd.service.AuthorService;
import com.wx.demo.wcd.service.impl.AuthorServiceImpl;

/**
 * 类名:AuthorCrawler <br/>
 * 创建时间: 2017年2月22日 下午3:56:20 <br/>
 * 〈功能简述〉古诗文网站诗人词人数据爬取. <br/>
 * 〈功能详细描述〉.<br/>
 * 
 * @author 王笑 Email:wangxiao@yijiehulian.com.
 * @version
 * @since JDK 1.8
 * @see
 */
public class AuthorCrawler extends BreadthCrawler {

	private Logger logger = Logger.getLogger(getClass());

	/**
	 * 读取Spring配置信息
	 */
	AbstractApplicationContext ctx = new ClassPathXmlApplicationContext(
			new String[] { "classpath*:/spring/spring-dao.xml",
					"classpath*:/spring/spring-mvc.xml",
					"classpath*:/spring/spring-service.xml" });
	AuthorService authorService = (AuthorServiceImpl) ctx
			.getBean("authorService");

	public static void main(String[] args) throws Exception {
		AuthorCrawler crawler = new AuthorCrawler("depth_crawler", true);
		// 总页面
		int ps = 3131;
		int t;
		if ((ps % 10) == 0) {
			t = (ps / 10);
		} else {
			if ((ps / 10) > 0) {
				t = (ps / 10) + 1;
			} else {
				t = 1;
			}
		}
		// 迭代生成需要爬取的URL
		for (int i = 1; i <= t; i++) {
			crawler.addSeed(new CrawlDatum(
					"http://so.gushiwen.org/authors/Default.aspx?p=" + i
							+ "&c=").meta("depth", "1"));
		}

		/* 线程数 */
		crawler.setThreads(50);
		/* 设置每次迭代中爬取数量的上限 */
		crawler.setTopN(t + 1);
		// 爬取深度
		crawler.start(3);
	}

	public AuthorCrawler(String crawlPath, boolean autoParse) {
		super(crawlPath, autoParse);
	}

	@Override
	public void visit(Page page, CrawlDatums next) {

		// 当前url
		String url = page.getUrl();
		logger.info("URL:\n" + url);

		// 有效节点信息
		Elements Dom = page.select("div[class=main3]");
		Elements ets = Dom.select("div[class=sonsauthor]");

		// 初始化作者对象集合
		ArrayList<AuthorPO> la = new ArrayList<AuthorPO>();

		for (Element et : ets) {
			AuthorPO author = new AuthorPO(); // 创建时间
			author.setAuthorId(UUIDGenerator.generate());// 作者表标识
			author.setCreateTime(new Date()); // 作者画像链接
			String imgSrc = et.select("img").attr("src");
			if (imgSrc != null && !"".equals(imgSrc)) {
				author.setPortraitLink(imgSrc);
			}
			logger.info("图像地址:" + et.select("img").attr("src"));

			Elements es = et.select("p");
			int j = 0;
			for (Element e : es) {
				j++;
				if (j == 1) {
					String name = e.text();// 姓名
					if (name != null && !"".equals(name)) {
						author.setAuthorName(name);
					}
				}
				if (j == 2) {
					String dynasty = e.text();// 朝代
					if (dynasty != null && !"".equals(dynasty)) {
						String dt = dynasty.substring(dynasty.indexOf("：") + 1,
								dynasty.length());
						if ("先秦".equals(dt)) {
							author.setDynasty("02");
						} else if ("两汉".equals(dt)) {
							author.setDynasty("03");
						} else if ("魏晋".equals(dt)) {
							author.setDynasty("04");
						} else if ("南北朝".equals(dt)) {
							author.setDynasty("05");
						} else if ("隋代".equals(dt)) {
							author.setDynasty("06");
						} else if ("唐代".equals(dt)) {
							author.setDynasty("07");
						} else if ("五代".equals(dt)) {
							author.setDynasty("08");
						} else if ("宋代".equals(dt)) {
							author.setDynasty("09");
						} else if ("金朝".equals(dt)) {
							author.setDynasty("10");
						} else if ("元代".equals(dt)) {
							author.setDynasty("11");
						} else if ("明代".equals(dt)) {
							author.setDynasty("12");
						} else if ("清代".equals(dt)) {
							author.setDynasty("13");
						} else if ("近代".equals(dt)) {
							author.setDynasty("14");
						} else {
							// 未知
							author.setDynasty("01");
						}
					}
				}
				if (j == 3) {
					String synopsis = e.text();// 简介
					if (synopsis != null && !"".equals(synopsis)) {
						author.setSynopsis(synopsis);
					}
				}
			}
			if (author.inputDescription() != null
					&& !"".equals(author.inputDescription())) {
				// 将对象添加到list集合
				la.add(author);
			}
		}

		try {
			int rows = authorService.insertBatch(la);
			if (rows > 0) {
				logger.info("数据抓取并存储成功！");
			} else {
				logger.info("数据抓取并存储失败！");
			}
		} catch (Exception e1) {
			e1.printStackTrace();
		}

	}
}
